#!/usr/bin/env python
# coding=utf-8
"""
@Author  : youjia - 卞志伟
@file    : city_home_spider.py
@contact : bianzhiwei@iyoujia.com
@time    : 2019-05-24 10:34 
@Desc    : 
@Software: PyCharm
"""
import os
import sys

# 当前文件的路径
pwd, filename = os.path.split(os.path.abspath(__file__))
# 当前文件的父路径
father_path = os.path.abspath(os.path.dirname(pwd) + os.path.sep + ".")
# 当前文件的前两级目录
grader_father = os.path.abspath(os.path.dirname(pwd) + os.path.sep + "..")
sys.path.append(pwd)
sys.path.append(father_path)
sys.path.append(grader_father)

import pymongo
import datetime
from report_system.spider.spider import Spider
from report_system.utils import db_util
from report_system.utils import date_util
from report_system.utils import mail_util
from report_system.utils.log_util import log
from report_system.utils.development_util import debug


class CityHome(Spider):
    def __init__(self):
        Spider.__init__(self, spider_num=30 if not debug else 20)
        self.name = 'cityhome'
        self.mongo_url = db_util.mongodb_test_url_ if debug else db_util.mongodb_url_
        self.mongo_client_ = None
        self.init_db()
        time_interval_ = '15:00'
        # time_interval_ = '19:00'
        # self.expire_ = {'all_city': [1, '00:10', '23:50'],
        #                 'city_data': [1, '00:10', '23:50'],
        #                 'list': [1, '00:10', '23:50'],
        #                 'detail': [1, '00:10', '23:50'],
        #                 'comment': [1, '00:10', '23:50'],
        #                 'calendar_daily': [1, '00:10', '23:50'],
        #                 'calendar': [1, '00:10', '23:50'],
        #                 'landlord': [1, '00:10', '23:50']}

        # self.expire_ = {'all_city': [23, '00:10', '23:50'],
        #                 'city_data': [23, '00:10', '23:50'],
        #                 'list': [12, time_interval_, '23:50'],
        #                 'detail': [10 * 24, '00:10', time_interval_],
        #                 'comment': [10 * 24, '00:10', time_interval_],
        #                 'calendar_daily': [12, time_interval_, '23:50'],
        #                 'calendar': [1, '00:10', '23:50'],
        #                 'landlord': [10 * 24, '00:10', time_interval_]}

    def con(self, set_name):
        """
        获取 mongodb 连接
        :param set_name:
        :return:
        """
        if self.mongo_client_ is None:
            self.mongo_client_ = pymongo.MongoClient(self.mongo_url)
        return self.mongo_client_[self.name][set_name]

    def init_db(self):
        log.info('初始化数据库 ...')
        self.con('city').create_index([('id', pymongo.ASCENDING)], unique=True)
        self.con('unit').create_index([('city_id', pymongo.ASCENDING)], unique=False)
        self.con('unit').create_index([('create_time', pymongo.ASCENDING)], unique=False)
        self.con('unit').create_index([('id', pymongo.ASCENDING)], unique=False)
        self.con('unit').create_index([('id', pymongo.ASCENDING), ('city_id', pymongo.ASCENDING)], unique=True)
        self.con('unit').create_index([('city_id', pymongo.ASCENDING), ('online', pymongo.ASCENDING)], unique=False)
        self.con('unit').create_index([('id', pymongo.ASCENDING), ('online', pymongo.ASCENDING)], unique=False)
        self.con('unit').create_index([('online', pymongo.ASCENDING)], unique=False)

        self.con('unit').create_index([('city_id', pymongo.ASCENDING), ('list_update_time', pymongo.ASCENDING)],
                                      unique=False)
        self.con('unit').create_index([('city_id', pymongo.ASCENDING), ('detail_update_time', pymongo.ASCENDING)],
                                      unique=False)
        self.con('unit').create_index(
            [('city_id', pymongo.ASCENDING), ('calendar_daily_update_time', pymongo.ASCENDING)], unique=False)
        self.con('unit').create_index([('city_id', pymongo.ASCENDING), ('calendar_update_time', pymongo.ASCENDING)],
                                      unique=False)
        self.con('unit').create_index([('city_id', pymongo.ASCENDING), ('comment_update_time', pymongo.ASCENDING)],
                                      unique=False)

        self.con('unit').create_index([('create_time', pymongo.ASCENDING)], unique=False)
        self.con('unit').create_index([('version_new', pymongo.ASCENDING)], unique=False)
        self.con('unit').create_index([('list_update_time', pymongo.ASCENDING)], unique=False)
        self.con('unit').create_index([('detail_update_time', pymongo.ASCENDING)], unique=False)
        self.con('unit').create_index([('calendar_daily_update_time', pymongo.ASCENDING)], unique=False)
        self.con('unit').create_index([('calendar_update_time', pymongo.ASCENDING)], unique=False)
        self.con('unit').create_index([('comment_update_time', pymongo.ASCENDING)], unique=False)
        self.con('unit').create_index([('landlord_update_time', pymongo.ASCENDING)], unique=False)

    # def expired(self, city_id, item, update_time_key):
    #     # 检查时间段
    #     if debug:
    #         return True
    #     cfg = self.expire_[item]
    #     start_t = datetime.datetime.strptime(cfg[1], '%H:%M').time()
    #     end_t = datetime.datetime.strptime(cfg[2], '%H:%M').time()
    #     now = datetime.datetime.now()
    #     now_t = now.time()
    #     if not start_t < now_t < end_t:
    #         log.info('%s 非工作时间段', item)
    #         return False
    #     col = self.con('city' if 'city' in item else ('landlord' if 'landlord' in item else 'unit'))
    #     item_id = 'id' if 'city' in item else 'city_id'
    #     online = {item_id: city_id, 'online': True} if 'all_city' not in item else {'online': True}
    #     empty = False if col.find_one(online) else True
    #     online['$or'] = [{update_time_key: {'$lt': self.expire_time(item)}}, {update_time_key: {'$exists': False}}]
    #     if item == 'calendar_daily':
    #         empty = False
    #         online['online'] = False
    #     expired = True if col.find_one(online) else False
    #     ret = empty or expired
    #     if not ret:
    #         log.info('%s 最新', item)
    #     return ret
    #
    # def expire_time(self, item):
    #     cfg = self.expire_[item]
    #     interval = datetime.timedelta(hours=cfg[0])
    #     now = datetime.datetime.now()
    #     return now - interval

    def all_city(self):
        """
        https://api.city-home.cn/common/cityhome_for_consumer/place_list/?extra_info=%7B%22place%22%3A7%7D&time=1558666788860
        城市基本信息爬取 榛果的城市列表在首页https://www.zhenguo.com/中 正则取
        :return:
        """
        time_stamp = date_util.time_stamp()
        url = 'https://api.city-home.cn/common/cityhome_for_consumer/place_list/?extra_info={"place":7}&time='
        log.info('请求国内城市列表 ...')
        self.req(url=url + time_stamp, method='GET', handler=self.all_city_h)

    def all_city_h(self, r, j):
        log.info('正在解析城市列表..')
        city_con = self.con('city')
        if r and r.status_code == 200 and j and 'status' in j and j['status'] == 0:
            for city in j['data']['domestic']:
                city_id = city['id']
                city_con.update_one({'id': int(city_id)}, {'$set': city}, upsert=True)
            self.req(handler=self.all_city_done)
        else:
            raise ValueError('城市列表出错')

    def all_city_done(self):
        log.info('爬取城市列表完成！')
        self.req(handler=self.city)

    def city(self, city_id=0):
        """

        :param city_id:
        :return:
        """
        log.info('开始循环城市进行爬取，请稍等。。。')
        con = self.con('city')
        city = con.find_one({'id': {'$gte': city_id}}, {'_id': 0})
        # print(city)
        if city is None:
            self.city_done()
            return
        self.req(handler=self.unit_list, city=city)

    def city_done(self):
        log.info(self.name + ' 爬虫所有城市列表爬去完毕！')

    def unit_list(self, city):
        """
        https://api.city-home.cn/wechat_service/v2/room/map/?is_available=true&city=1&extra_info=%7B%22place%22%3A1%7D&time=1558676613687
        :param city:
        :return:
        """
        log.info('开始-爬取-城市： {} 下的所有房源信息。请稍等。。。'.format(city['name']))
        room_list_url = 'https://api.city-home.cn/wechat_service/v2/room/map/?' \
                        'is_available=true&city={city}&extra_info={extra_info}' \
                        '&time={time}'.format(city=city['id'], time=date_util.time_stamp(), extra_info={"place": 1})

        self.req(handler=self.unit_list_h, method='GET', url=room_list_url, city=city)

    def unit_list_h(self, r, j, city):
        """
        解析城市的房源列表
        :param r:
        :param j:
        :param city:
        :return:
        """
        log.info('开始-解析-城市： {} 下的所有房源信息。请稍等。。。'.format(city['name']))
        if r and r.status_code == 200 and j and 'status' in j and j['status'] == 0:
            city_id = city['id']
            city_name = city['name']
            total_count = j['data']['total_count']
            con = self.con('unit')

            for idx, unit in enumerate(j['data']['room_list']):
                unit['city_id'] = city_id
                unit['city_name'] = city_name
                con.update_one({'city_id': int(city_id), 'id': unit['room_id']},
                               {'$set': {'online': True,
                                         'city_name': city_name,
                                         'city_total_count': total_count,
                                         'list_order.' + date_util.today_str(): idx + 1,
                                         'list_update_time': date_util.datetime_now(),
                                         'list': unit}}
                               , upsert=True)
                con.update_many({'city_id': city_id, 'id': unit['room_id'], 'create_time': {'$exists': False}},
                                {'$set': {'create_time': date_util.datetime_now()}})
            self.req(handler=self.unit_list_done, city=city)
        else:
            raise ValueError('解析城市的房源列表 出错！')

    def unit_list_done(self, city):
        # log.info('爬取-城市： {} 下的所有房源信息 完成 。。。'.format(city['name']))
        # if not self.expired(city['id'], 'detail', 'detail_update_time'):
        #     log.info('不是爬取详情页的时间。。。。')
        #     self.detail_done(city)
        # else:
        log.info('开始 爬取-城市： {} 下的所有房源 详情页   。。。'.format(city['name']))
        con = self.con('unit')
        units = con.find({'city_id': city['id'],
                          'online': True,
                          'id': {'$ne': None}
                          # ,'$or': [{'detail_update_time': {'$exists': False}},
                          #         {'detail_update_time': {'$lt': self.expire_time('detail')}}]
                          }, {'id': 1})
        for unit in units:
            self.detail(city=city, unit_id=unit['id'])
        self.req(handler=self.detail_done, city=city)

    def detail(self, city, unit_id):
        """
        准备访问详情页
        :param city:
        :param unit_id:
        :return:
        """
        log.info('开始 -- 处理城市 {city} 的 房源 {unit_id} 详情页'.format(city=city['name'], unit_id=unit_id))
        url = 'https://api.city-home.cn/wechat_service/v2/room/{unit_id}/?time=1558681508753'.format(unit_id=unit_id)
        self.req(handler=self.detail_h, url=url, city=city, unit_id=unit_id)

    def detail_h(self, r, j, city, unit_id):
        """
        处理详情页
        :param r:
        :param j:
        :param city:
        :param unit_id:
        :return:
        """
        if r and r.status_code == 200 and j and 'status' in j and j['status'] == 0 and 'data' in j:
            log.info('正在 -- 存储城市 {city} 的 房源 {unit_id} 详情页'.format(city=city['name'], unit_id=unit_id))
            self.con('unit').update_one({'city_id': city['id'], 'id': unit_id},
                                        {'$set': {'detail': j['data'],
                                                  'detail_update_time': date_util.datetime_now()}},
                                        upsert=True)

    def detail_done(self, city):
        """
        :param city:
        :return:
        """
        log.info('处理城市 {city} 的 房源 详情页 -- 完成'.format(city=city['name']))
        # if not self.expired(city['id'], 'calendar', 'calendar_update_time'):
        #     log.info('不是处理房源日历的时间。。。。')
        #     self.calendar_done(city['id'])
        # else:
        log.info('开始爬取城市 {city} 的 房源日历  '.format(city=city['name']))
        con = self.con('unit')
        units = con.find({'city_id': city['id']}, {'id': 1})
        for unit in units:
            self.calendar(city=city, unit_id=unit['id'])
        self.req(handler=self.calendar_done, city=city)

    def calendar(self, city, unit_id):
        log.info('开始 -- 处理城市 {city} 的 房源 {unit_id} 日历'.format(city=city['name'], unit_id=unit_id))
        st = date_util.today_str()
        et = date_util.today_str(date_util.curdate(-100))
        time_stamp = date_util.time_stamp()
        url = "https://api.city-home.cn/orders/cityhome_price_calendar/?" \
              "start_date={st}&end_date={et}&room_id={unit_id}" \
              "&extra_info=%7B%22place%22%3A1%7D&time={time}".format(st=st, et=et, unit_id=unit_id, time=time_stamp)
        # print(url)
        self.req(handler=self.calendar_h, url=url, city=city, unit_id=unit_id)

    def calendar_h(self, r, j, city, unit_id):
        if r and r.status_code == 200 and j and 'status' in j and j['status'] == 0 and 'data' in j:
            curdate = date_util.curdate(0)
            log.info('开始 -- 存储城市 {city} 的 房源 {unit_id} 日历'.format(city=city['name'], unit_id=unit_id))
            con = self.con('unit')
            room_list = j['data']['room_list']
            if room_list.__len__() == 0:
                log.warning('房源 {unit_id} 的价格日历为空。。'.format(unit_id=unit_id))
            for room in room_list:
                room_id = room['room_id']
                order_list = room['order_list'] if 'order_list' in room else []
                # 将今天的订单初始化 置为0
                have_order = 0
                for book_order in order_list:
                    checkin_date = book_order['checkin_date']
                    checkout_date = book_order['checkout_date']
                    platform = book_order['platform']
                    book_order_id = str(checkin_date) + '_' + str(checkout_date) + '_' + str(platform)
                    con.update({'city_id': city['id'], 'id': room_id},
                               {'$set': {'book_order.' + book_order_id: book_order}}, upsert=True)
                    st = date_util.str_to_date(checkin_date)
                    et = date_util.str_to_date(checkout_date)

                    if st <= curdate < et:
                        # 如果有订单 将今天的订单加1
                        have_order += 1
                con.update({'city_id': city['id'], 'id': room_id},
                           {'$set': {'lodge_calendar.' + str(curdate) + '.have_order': have_order}}, upsert=True)

                for price_dict in room['price_list']:
                    date = price_dict['date']
                    con.update({'city_id': city['id'], 'id': room_id},
                               {'$set': {'calendar.' + date + '.price': price_dict['price']}}, upsert=True)

                    if str(date) == str(curdate):
                        con.update({'city_id': city['id'], 'id': room_id},
                                   {'$set': {'lodge_calendar.' + str(curdate) + '.price': price_dict['price']}},
                                   upsert=True)

            con.update_many({'city_id': city['id'], 'id': unit_id},
                            {'$set': {'calendar_update_time': datetime.datetime.now()}})

    def calendar_done(self, city):
        """

        :param city:
        :return:
        """
        log.info('处理城市 {city} 的 房源日历 -- 完成'.format(city=city['name']))
        # if not self.expired(city['id'], 'comment', 'comment_update_time'):
        #     log.info('不是处理评论的时间...')
        #     self.comment_done(city)
        # else:
        log.info('开始处理城市 {city} 的 评论 '.format(city=city['name']))
        con = self.con('unit')
        units = con.find({'city_id': city['id'],
                          'online': True
                          # ,'$or': [{'comment_update_time': {'$exists': False}},
                          #         {'comment_update_time': {'$lt': self.expire_time('comment')}}]
                          },
                         {'id': 1})
        for unit in units:
            self.comment(city=city, unit_id=unit['id'])
        self.req(handler=self.comment_done, city=city)

    def comment(self, city, unit_id):
        """
        https://api.city-home.cn/rooms/cityhome/2808555435/reviews/?cursor=0&count=20&extra_info=%7B%22place%22%3A1%7D&time=1558684418439
        :param city:
        :param unit_id:
        :return:
        """
        url = "https://api.city-home.cn/rooms/cityhome/{unit_id}/reviews/?" \
              "cursor=0&count=1000&extra_info=%7B%22place%22%3A1%7D&time={time}".format(unit_id=unit_id,
                                                                                        time=date_util.time_stamp())
        self.req(handler=self.comment_h, url=url, city=city, unit_id=unit_id)

    def comment_h(self, r, j, city, unit_id):
        if r and r.status_code == 200 and j and 'status' in j and j['status'] == 0 and 'data' in j:
            log.info('开始 -- 存储城市 {city} 的 房源 {unit_id} 评论'.format(city=city['name'], unit_id=unit_id))
            self.con('unit').update_one({'city_id': city['id'], 'id': unit_id},
                                        {'$set': {'comment': j['data']['reviews'],
                                                  'comment_update_time': date_util.datetime_now()}},
                                        upsert=True)

    def comment_done(self, city):
        log.info('处理城市 {city} 的 房源评论 -- 完成'.format(city=city['name']))
        # if not self.expired(city['id'], 'landlord', 'hotel_update_time'):
        #     # 下一个城市爬取， 避免递归深度异常
        #     log.info('不是处理房东信息的时间。。。。')
        #     self.landlord_done(city)
        # else:
        log.info('开始处理城市 {city} 的 房东信息 '.format(city=city['name']))
        con = self.con('unit')
        units = con.find({'city_id': city['id'],
                          'online': True
                          # ,'$or': [{'landlord_update_time': {'$exists': False}},
                          #         {'landlord_update_time': {'$lt': self.expire_time('landlord')}}]
                          }, {'id': 1})
        for unit in units:
            self.landlord(city=city, unit_id=unit['id'])
        self.req(handler=self.landlord_done, city=city)

    def landlord(self, city, unit_id):
        url = "https://api.city-home.cn/weixinmp/landlord/12/?room_id={unit_id}" \
              "&extra_info=%7B%22place%22%3A1%7D&time={time}".format(unit_id=unit_id, time=date_util.time_stamp())

        self.req(handler=self.landlord_h, url=url, city=city, unit_id=unit_id)

    def landlord_h(self, r, j, city, unit_id):
        if r and r.status_code == 200 and j and 'status' in j and j['status'] == 0 and 'data' in j:
            log.info('开始 -- 存储城市 {city} 的 房源 {unit_id} 房东信息'.format(city=city['name'], unit_id=unit_id))
            self.con('unit').update_one({'city_id': city['id'], 'id': unit_id},
                                        {'$set': {'landlord': j['data'],
                                                  'landlord_update_time': date_util.datetime_now()}},
                                        upsert=True)

    def landlord_done(self, city):
        log.info('处理城市 {city} 的 房东信息 -- 完成    '.format(city=city['name']))
        # 下一个城市爬取， 避免递归深度异常
        log.info('开始处理下一个城市。。。。')
        self.req(handler=self.city, city_id=int(city['id']) + 1)

    def run(self):
        self.all_city()
        Spider.run(self)


def main():
    # while 1:
    log.info('途家爬虫运行 ...')
    CityHome().run()
    # time.sleep(60 * 30)
    # 重启程序
    # util.restart_program()


def test():
    import requests
    url = "https://api.city-home.cn/orders/cityhome_price_calendar/?start_date=2019-01-21&end_date=2020-04-30&room_id=2808555383&extra_info=%7B%22place%22%3A1%7D&time=1558688910471"
    j = requests.get(url).json()
    room_list = j['data']['room_list']
    # if room_list.__len__() == 0:
    #     log.warning('房源 {unit_id} 的价格日历为空。。'.format(unit_id=unit_id))
    con = CityHome().con('unit')
    for room in room_list:
        room_id = room['room_id']
        order_list = room['order_list'] if 'order_list' in room else []
        for book_order in order_list:
            checkin_date = book_order['checkin_date']
            checkout_date = book_order['checkout_date']
            platform = book_order['platform']
            book_order_id = str(checkin_date) + '_' + str(checkout_date) + '_' + str(platform)
            print(book_order_id, room_id, 2808555383)
            print(book_order)
            con.update({'city_id': 1, 'id': room_id},
                       {'$set': {'book_order.' + book_order_id: book_order}}, upsert=True)


if __name__ == '__main__':
    try:
        main()
    except Exception as e:
        mail_util.error_mail("竞对下午房源小时爬虫", e.__str__())
        log.exception(e)
